from collections import defaultdict
import os,sys
os.chdir(os.path.dirname(__file__))

seprator = "========================================================================"
info = defaultdict(int)
instances = []
pre_words = ["[PAD]","[UNK]","[CLS]","[SEP]","[MASK]"]
words = set()
final_words = []
labels = set()
def make_vocab(file = "1.txt"):
    with open(file=file,mode='r',encoding="GB18030")as f:
        lines = f.read()
        raw_instances = lines.split(seprator)
        for raw_instance in raw_instances:
            raw_instance_str = raw_instance.strip()

            if raw_instance_str:#最后一行可能是空的换行符
                single_instance_dict = defaultdict(list)
                instance_pre = raw_instance_str.split("\n")

                #公式得到
                single_instance_dict[instance_pre[0][:3]] = instance_pre[0][3:]
                #标签得到
                label = instance_pre[-1][3:]
                single_instance_dict[instance_pre[-1][:3]] = label
                labels.add(label)
                layers_pre = instance_pre[1:-1]

                #记录一下最大层数
                if len(layers_pre)>info["max_layers"]:
                    info["max_layers"] = len(layers_pre)

                #格式化
                for ix,layer_str in enumerate(layers_pre):
                    layer_ix_str = layer_str[len(f"第{ix}层："):]
                    if layer_ix_str:
                        layer_words = [word for word in layer_ix_str.split("#") if word]
                        single_instance_dict["layers"].append(layer_words)
                        words.update(layer_words)
                    if "" in words:
                        raise

                    #记录一下最大一层的分割数
                    if len(layer_words) > info["max_words"]:
                        info["max_words"] = len(layer_words)
            instances.append(single_instance_dict)
        final_words.extend(pre_words + list(words))
        if __name__ == "__main__":
            print(len(instances))
        labels2id = {label:ix for ix,label in enumerate(labels)}
        with open("vocab.txt",'w')as vocab_f:
            vocab_f.write("\n".join(final_words))
    return {"instances":instances,
            "final_words":final_words,
            "labels2id":labels2id,
            "max_layers":info["max_layers"],
            "max_words":info["max_words"],
            }

if __name__ == "__main__":
    print(make_vocab()["instances"].__len__())